"""
sliplib.py - SLiP ("Sorta Like Python") shorthand to XML conversion library

Last Updated: 
	
	$Date: 2002/04/19 20:41:13 $

License: 

	MIT License

	Copyright (c) 2002 Scott Sweeney
	
	Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
	documentation files (the "Software"), to deal in the Software without restriction, including without limitation 
	the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 
	and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
	
	The above copyright notice and this permission notice shall be included in all copies 
	or substantial portions of the Software.
	
	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING 
	BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
	NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
	DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


Description:
					 
	This module exposes two functions called 'slip2xml()' and 'xml2slip()'.  They convert
	Pythonish-formatted data to XML and back.


	slip2xml()
		Input(s) - strSLIP: a sting of SLIP-formatted text
		Output(s) - strXML: a string of XML

	xml2slip()
		Input(s) - strXML: a string of XML
		Output(s) - strSLIP: a sting of SLIP-formatted text


To Do:
	Remove unnecessary triple quotes
	Support internal DTDs
	Solidify API


"""

__version__ = '0.3'

import sys
import re
import os
#import tokenize
import sliptokenize #need custom tokenize for comment handling
import string, StringIO #needs to be StringIO, not cStringIO due to unicode

from xml.parsers import expat


ELEMENTNAME = 1
ATTRNAME = 2

strText = u""
strDTD = u""
strXML = u""
tokens = []
intIndentLevel = 0
dictLevelName = {}
dictEscapedChars = {}
tupQuoteOn = (0, u"")
strWhitespaceCache = ""
booCDataOn = 0
strUnicodeXML = u""
lsLines = []
strCurrentSLIPString = u""

booHasInternalDTD = 0


class EncodingError(Exception):
	pass
	
class TokenError(Exception):
	pass


def cycleNameValues(tupNameValue):
	strXML = u""
	strXML += u"<" + tupNameValue[0] + u">"
	lsTupGroups = reAll.findall(tupNameValue[1])

	if (lsTupGroups == []):
		strXML += tupNameValue[1]
	else:
		for tup in lsTupGroups:
			strXML += cycleNameValues(tup)

	strXML += u"</" + tupNameValue[0] + u">"
	return strXML
	
	
def backtrackForBeginning(intIndex):
	#in case it's the first element, but some whitespace or comments are before the first element
	while 1:
		if intIndex == 0:
			booBeginningFound = 1
			break
		else:
			tupToken = tokens[intIndex-1]
			strTokenType = sliptokenize.tok_name[tupToken[0]]
			if strTokenType == "COMMENT" or strTokenType == "NL" or strTokenType == "NEWLINE" or strTokenType == "INDENT" or strTokenType == "DEDENT" or strTokenType == "STRING": #STRING is for XML declarations
				intIndex -= 1
			elif strTokenType == "ERRORTOKEN" and tupToken[1] == u"\ufeff":
				#Some documents will have this Byte Order Mark
				#.. if you have this, you are at the beginning
				booBeginningFound = 1
				break
			else:
				booBeginningFound = 0
				break
				
	return booBeginningFound

def backtrackForIndentOrDedent(intIndex):
	#special function for case of mixed text and tags in an element's content, with text first
	#starting at NAME token, work backwards to verify that there is just text inbetween the NAME and the INDENT
	#or also a DEDENT
	while 1:
		tupToken = tokens[intIndex-1]
		strTokenType = sliptokenize.tok_name[tupToken[0]]
		if strTokenType == "NEWLINE" or strTokenType == "STRING" or strTokenType == "NL" or strTokenType == "COMMENT":
			intIndex -= 1
		elif strTokenType == "INDENT" or strTokenType == "DEDENT": #BUGFIX: Also need dedent clause here for mixed text
			booIndentFound = 1
			break
		else:
			booIndentFound = 0
			break
		
	return booIndentFound
	

def backtrackForName(intIndex):
	#for empty tags
	#ex. 
	#     element1():
	#     element2():
	

	while 1:
		tupToken = tokens[intIndex-1]
		strTokenType = sliptokenize.tok_name[tupToken[0]]
		if strTokenType == "NEWLINE" or strTokenType == "STRING" or strTokenType == "NL" or strTokenType == "COMMENT":
			intIndex -= 1
		elif strTokenType == "INDENT":
			booIndentlessNameFound = 0
			break
		#elif strTokenType == "OP" and tupToken[1] == ":" and tokens[intIndex-2][1] == ")":
		elif strTokenType == "OP" and tupToken[1] == ":" and (sliptokenize.tok_name[tokens[intIndex][0]] == "NL" or sliptokenize.tok_name[tokens[intIndex][0]] == "NEWLINE" or sliptokenize.tok_name[tokens[intIndex][0]] == "STRING"):
			booIndentlessNameFound = 1
			break
		else:
			booIndentlessNameFound = 0
			break

	return booIndentlessNameFound	



		
def stripQuotes(strQuoted):

	#won't eval unicode strings while in unicode, so temporarily encode as UTF-8
	#NOTE: may cause some problems, but good enough for now
	strUTF8 = strQuoted.encode("utf-8")
	strUnquoted = eval(strUTF8)
	strUnquoted = unicode(strUnquoted, "utf-8")


	#~ strUnquoted = ""
	#~ #dictEscapes = {"\n":}
	#~ if strQuoted[:3] == u"\"\"\"" and strQuoted[-3:] == u"\"\"\"":
		#~ strUnquoted = strQuoted[3:-3]
	#~ elif strQuoted[:1] == u"\"" and strQuoted[-1:] == u"\"":
		#~ strUnquoted = strQuoted[1:-1]
	#~ else:
		#~ #raise UnquotingError, "Could not unquote"
		#~ print "UnquotingError: Could not unquote"
		
		
	#~ #for strEscape in dictEscapes.keys():
	#~ #	pass
	
	return strUnquoted
	
	

def tokenHandler(type, token, (srow, scol), (erow, ecol), line):
	global intIndentLevel
	tokens.append((type, token, (srow, scol), (erow, ecol), line, intIndentLevel))

	#print "%d,%d-%d,%d:\t%s\t%s\tIndentLvl:%s" % \
	#	(srow, scol, erow, ecol, sliptokenize.tok_name[type], repr(token), intIndentLevel) #DEBUG

	if sliptokenize.tok_name[type] == "INDENT":
		intIndentLevel += 1
	elif sliptokenize.tok_name[type] == "DEDENT":
		intIndentLevel -= 1
		



def tokens2xml(lsTokens):
	global dictLevelName
	strXML = ""
	tokenCount = 0
	commentOn = 0
	
	for tupToken in lsTokens:
		listIndex = tokenCount
		type = tupToken[0]
		token = tupToken[1]
		srow = tupToken[2][0]
		scol = tupToken[2][1]
		erow = tupToken[3][0]
		ecol = tupToken[3][1]
		line = tupToken[4]
		indent = tupToken[5]
		

		
		if sliptokenize.tok_name[type] == "NAME":

			#check to see if it is an element name, based on proximity to indent or dedent or
			#that only text comes before the the indent, ignore all others (they are proactively handled) 
			
			#print token
			#print "Beginning? " + repr(backtrackForBeginning(listIndex))
			#print "Indent or Dedent? " + repr(backtrackForIndentOrDedent(listIndex))
			#print "Name? " + repr(backtrackForName(listIndex))


			if backtrackForBeginning(listIndex) == 1 or \
			   sliptokenize.tok_name[lsTokens[listIndex-1][0]] == "INDENT" or \
			   sliptokenize.tok_name[lsTokens[listIndex-1][0]] == "DEDENT" or \
			   backtrackForIndentOrDedent(listIndex) == 1 or \
			   backtrackForName(listIndex) == 1:
				
				
				#need to accomidate tokens in tagname (-, .) as well as namespaces
				currentToken = listIndex
				strName = ""
				booNoAttributes = 0
				
				
				while 1:
					if lsTokens[currentToken][1] == "(" :
						break
					elif lsTokens[currentToken][1] == ":" and (sliptokenize.tok_name[lsTokens[currentToken+1][0]] == "NL" or sliptokenize.tok_name[lsTokens[currentToken+1][0]] == "NEWLINE" or sliptokenize.tok_name[lsTokens[currentToken+1][0]] == "STRING"):
						#allows for element declarations without attributes to not need parens
						booNoAttributes = 1
						break
					elif sliptokenize.tok_name[lsTokens[currentToken][0]] == "OP" and lsTokens[currentToken][1] in u"-.:_":
						strName += lsTokens[currentToken][1]
						currentToken = currentToken + 1
					elif sliptokenize.tok_name[lsTokens[currentToken][0]] == "NAME":
						strName += lsTokens[currentToken][1]
						currentToken = currentToken + 1
					else:
						raise TokenError, "Unallowed or unrecognized token - " + lsTokens[currentToken][1]
						break
					
				
				elemName = strName
				strXML += u"<" + strName
				
				#attrIndex keeps track of where to start looking for attributes	
				#attrIndex = currentToken -1	
				
				
				strAttr = ""
				#seek attributes
				if not booNoAttributes:
					attrIndex = currentToken + 1 #token after left paren or colon
					while 1:
						if sliptokenize.tok_name[lsTokens[attrIndex][0]] == "NAME" or (sliptokenize.tok_name[lsTokens[attrIndex][0]] == "OP" and lsTokens[attrIndex][1] in u"=:-._"):
							strAttr += lsTokens[attrIndex][1]
							attrIndex += 1
						elif lsTokens[attrIndex][1] == ",":
							strAttr += u" "
							attrIndex += 1
						elif sliptokenize.tok_name[lsTokens[attrIndex][0]] == "STRING":
							strAttr += u"\"" + stripQuotes(lsTokens[attrIndex][1]) + u"\""
							attrIndex += 1
						elif lsTokens[attrIndex][1]  == u")":
							break
						else:
							print "AttributeError"
							break #what happened??
					
					
					if string.strip(strAttr) != "":
						strXML += " " + strAttr
					
					newIndex = attrIndex +1 
					
				else:
					newIndex = currentToken + 1 

				#dictionary for tracking element names and the corresponding indent level
				dictLevelName[indent] = elemName
				lsStringsFound = []
				
				
		
				#now process the content after the declaration for peers/siblings that won't have indents or dedents
				while 1:
					try:
						strTokenName = sliptokenize.tok_name[lsTokens[newIndex][0]]
					except:
						#no next token, aka end of doc, so close it
						strXML += u"></" + dictLevelName[indent] + u">"
						break
						
					if strTokenName == "INDENT":
						strXML += u">"
						#found indent, do nothing
						break
					elif (strTokenName == "STRING" and sliptokenize.tok_name[lsTokens[newIndex-1][0]] != "NL") and (strTokenName == "STRING" and sliptokenize.tok_name[lsTokens[newIndex-1][0]] != "NEWLINE") :
						#found string, save for later in case we need it
						lsStringsFound.append(lsTokens[newIndex][1])
						newIndex += 1

					elif strTokenName == "NAME" or strTokenName == "DEDENT" or strTokenName == "COMMENT" or \
						(strTokenName == "STRING" and (sliptokenize.tok_name[lsTokens[newIndex-1][0]] == "NL" or sliptokenize.tok_name[lsTokens[newIndex-1][0]] == "NEWLINE")): #TODO: should this also handle for COMMENTs?
						#found name, print string and close element
						if len(lsStringsFound) != 0:
							strXML += u">"	
							for str in lsStringsFound:
								strXML += stripQuotes(str)
							strXML += u"</" + dictLevelName[indent] + u">"
						else:
							strXML += u"/>"
						break
					else:
						newIndex += 1
					
				

		elif sliptokenize.tok_name[type] == "COMMENT":
			if commentOn == 0:
				commentOn = 1
				strXML += u"<!--"
			if sliptokenize.tok_name[lsTokens[listIndex+1][0]] == "COMMENT":
				strXML += token[1:] #removed the '#' sign and left the new line
			elif sliptokenize.tok_name[lsTokens[listIndex+1][0]] != "COMMENT":
				if token[-1:] == "\n":
					strXML += token[1:-1] #removed the '#' sign and removed the new line
				else:
					strXML += token[1:] #removed the '#' sign 
				strXML += u"-->"
				commentOn = 0

		elif sliptokenize.tok_name[type] == "STRING" and sliptokenize.tok_name[lsTokens[listIndex-1][0]] != "OP":
			strXML += stripQuotes(token)
		elif sliptokenize.tok_name[type] == "DEDENT":
			strXML += u"</" + dictLevelName[indent-1] + u">"
			
		tokenCount += 1

	return strXML		

	
	


def slip2xml(strText):
	"""
	given a string of SLiP, return a str of xml
	"""
	global tokens, dictLevelName, tupQuoteOn, strXML
	tokens = []
	dictLevelName = {}
	tupQuoteOn = (0, u"")
	strXML = u""
	
	
	try:
		strUnicodeSLIP = unicode(strText, "ascii")
	except:
		strUnicodeSLIP = strText
		
	#TODO: verify that it is unicode or ascii
	#try:
	#	strUTF8SLIP = strUnicodeSLIP.encode("utf-8")
	#except:
	#	print "EncodingError: slip2xml() requires ascii or unicode string"

	
	fd = StringIO.StringIO(strUnicodeSLIP)
	sliptokenize.tokenize(fd.readline, tokeneater=tokenHandler)   
	strXML = tokens2xml(tokens)
	
	return strXML
	

def dumpWhitespaceCache():
	global strWhitespaceCache, strText
	
	#remove harmful whitespace (spaces and tabs) and print the new lines for nicer, more accurate formatting 
	#strNewlines = string.replace(strWhitespaceCache, "\t", "")
	#strNewlines = string.replace(strNewlines, " ", "")
	#strText += strNewlines
	
	#reset cache
	strWhitespaceCache = ""


def ensureQuoteOff():
	global strText, tupQuoteOn, intIndentLevel, strWhitespaceCache
	
	
	if tupQuoteOn[0] == 1:
	
		# Quadruple quote problem - if string ends with a quote and is triple-quoted, the parser only picks up the first 3 quotes... 
		# as result, need to escape the last quote
		
		# Note: the XML quoting could also be XML escaping rather than the Python escaping 
		# but since quotes are legal as unescaped as XML text, this could change some documents, although they will still be legal XML
	
		#if strText[-1:] == "\"": 
			#strText = strText[:-1] + "&quot;"
			#strText += r"\""
			
		strText += tupQuoteOn[1] + u"\n\n"
		tupQuoteOn = (0, u"")

		


def startElement(name, attrs):
	global strText, intIndentLevel, parser, lsLines
	
	intLineNo = parser.ErrorLineNumber
	intColNo = parser.ErrorColumnNumber

	ensureQuoteOff()	
	dumpWhitespaceCache()
	
	strText += (intIndentLevel * u"\t") + name + u"("

	for attr in attrs.items():
		strText += reescape(attr[0]) + u"=\"" + reescape(attr[1]) + u"\","

	if len(attrs) != 0:
		strText = strText[:-1] #get rid of last comma
		
	strText += u"):" 
		
	
	#detect if empty element or a set of start and elements without content
	#if empty set, render as empty string
	
	#get that line
	line = lsLines[intLineNo -1]
	intEndBracket = string.find(line, ">", intColNo)

	if intEndBracket + 1 != len(line) and line[intEndBracket+1:][:2] == u"</" and line[intEndBracket-1] != u"/": 
		#was a empty start and end element set; render with empty string ("")
		strText += u" \"\"\n"
	else:
		#was an empty element or element with some content
		strText += u"\n"
	
	intIndentLevel += 1
	
	
def endElement(name):
	global strText, intIndentLevel
	
	ensureQuoteOff()	
	dumpWhitespaceCache()

	strText += u"\n"
	intIndentLevel -= 1	

def charData(data):
	global strText, intIndentLevel, tupQuoteOn, strWhitespaceCache
	
	
	if not booCDataOn:
		data = reescape(data)
		data = string.replace(data, "\\", "\\\\")
		data = string.replace(data, "\"", "\\\"") #replace all quotes to be on the safe side (single quoted strings)

	strQuote = u"\"\"\""

	#if just whitespace, only include it if there is text also before or after it
	#... store it for the time being
	if string.strip(data) == "" and tupQuoteOn[0] == 0:
		strWhitespaceCache += data
	
	if string.strip(data) != "" and tupQuoteOn[0] == 0:
		#include any stored whitespace, but only the new lines, not tabs or spaces
		strNewlines = string.replace(strWhitespaceCache, "\t", "")
		strNewlines = string.replace(strNewlines, " ", "")
		
		strText += (intIndentLevel * u"\t") + strQuote + strNewlines + data 
		#strText += (intIndentLevel * u"\t") + strQuote + data 
		
		strWhitespaceCache = "" #not dumpWhitespaceCache because we want the raw whitespace
		tupQuoteOn = (1, strQuote)
		
	elif tupQuoteOn[0] == 1:
		strText += data
	

def startCData():
	global strText, intIndentLevel, tupQuoteOn
	
	dumpWhitespaceCache()
	booCDataOn = 1
	
	strQuote = u"\"\"\""	  
	if tupQuoteOn[0] == 0:
		strText += (intIndentLevel * u"\t") + strQuote + u"<![CDATA["
		tupQuoteOn = (1, strQuote)
	elif tupQuoteOn[0] == 1:
		strText += u"<![CDATA["

def endCData():
	global strText, tupQuoteOn

	booCDataOn = 0
	
	strText += u"]]>"
	#leave open for mixed content (ex. CDATA and string)


def comment(data):
	global strText, intIndentLevel, strWhitespaceCache
	ensureQuoteOff()
	dumpWhitespaceCache()

	strIndent = intIndentLevel * u"\t"   
	data = string.replace(data, u"\n", u"\n" + strIndent + u"#")
	#data = string.replace(data, "\\", "\\\\")
	strText += strIndent + u"#" + data + u"\n"
	
	#could also work like this...
	#strText += strIndent + u"\"\"\"<!-- " + data + u"-->\"\"\"\n"
	
def procInstruction(target, data):
	global strText, intIndentLevel
	ensureQuoteOff()
	dumpWhitespaceCache()
	
	strIndent = intIndentLevel * u"\t"   
	data = string.replace(data, "\\", "\\\\")
	strText += strIndent + u"\"\"\"<?" + target + " " + data + "?>\"\"\"\n"
	
	

def xmlDecl(version, encoding, standalone) :
	global strText

	strText = u"\"\"\"<?xml version=\"" + version + u"\" "
	if encoding:
		strText += u"encoding=\"" + encoding + u"\" "
	if standalone == 0:
		strText += u"standalone=\"no\" "
	elif standalone == 1:
		strText += u"standalone=\"yes\" "
	strText += u"?>\"\"\"\n"
	
	
def startDTD(doctypeName, systemId, publicId, has_internal_subset):
	global strText, strDTD, booHasInternalDTD
	

	strDTD += "\"\"\"<!DOCTYPE " + doctypeName + " "
	
	if systemId and not publicId:
		strDTD += "SYSTEM \"" + systemId + "\" " 
		
	if publicId and not systemId:
		strDTD += "PUBLIC \"" + publicId + "\" " 
		
	if has_internal_subset:
		sys.stderr.write("Internal DTDs are currently not supported at this time.\n")
		sys.stderr.write("	Internal DTD markup will be ignored.\n")
		#strDTD += "["
		#booHasInternalDTD = 1
	else:
		booHasInternalDTD = 0
	


def endDTD():
	global strText, strDTD, booHasInternalDTD 
	#if booHasInternalDTD:
	#	strDTD += "]"
	
	strDTD += ">\"\"\"\n"
	
	strText += strDTD


def elementDecl(name, model):
	pass
	
def attrDecl(elname, attname, type, default, required):
	pass
	
def entityDecl(entityName, is_parameter_entity, value, base, systemId, publicId, notationName):
	#print entityName, is_parameter_entity, value, base, systemId, publicId, notationName
	pass
	
def notationDecl(notationName, base, systemId, publicId):
	pass
	
def defaultHandler(data):
	pass


def reescape(str):
	global dictEscapedChars
	#special case - handle the & so it does not "over-escape" the string
	if string.find(str, u"&") != -1:
		reAmpr = re.compile(ur"&(?!(?:#|\w){2,7};)", re.U | re.I)
		str = re.sub(reAmpr, u"&amp;", str)
		
	#handle the rest
	for item in dictEscapedChars.items():
		str = string.replace(str, item[0], item[1])
		
	return str
	

def xml2slip(strXML):
	#should come in as python ascii string or unicode
	global strText, strUnicodeXML, lsLines
	strText = u""
	global dictEscapedChars
	dictEscapedChars = {}

	
	
	try:
		strUnicodeXML = unicode(strXML, "ascii")
	except:
		strUnicodeXML = strXML
		
	lsLines = string.split(strUnicodeXML, u"\n")


	try:
		strUTF8XML = strUnicodeXML.encode("utf-8")
	except:
		raise EncodingError, "xml2slip() requires ascii or unicode string"
		

	
	#need to search
	import re
	reXMLEscapedChars = re.compile(ur"&(?:#|\w){2,7};", re.U | re.I)
	lsEscapedChars = reXMLEscapedChars.findall(strUnicodeXML)
	for strEscapedChar in lsEscapedChars:
		if strEscapedChar == u"&amp;":
			#strUnescapedChar = u"&"
			continue #special case handled by reescape()
		elif strEscapedChar == u"&lt;":
			strUnescapedChar = u"<"
		elif strEscapedChar == u"&gt;":
			strUnescapedChar = u">"
		elif strEscapedChar == u"&quot;":
			strUnescapedChar = u"\""
		elif strEscapedChar == u"&apos;":
			strUnescapedChar = u"'"
		elif strEscapedChar[:3] == u"&#x":
			intFromHex = int("0x" + strEscapedChar[3:-1], 16)
			strUnescapedChar = unichr(intFromHex)
		elif strEscapedChar[:2] == u"&#":
			intFromStr = int(strEscapedChar[2:-1])
			strUnescapedChar = unichr(intFromStr)
		else:
			sys.stderr.write("EscapingWarning: Could not translate escaped char: " + strEscapedChar + "\n")
			sys.stderr.write("	Likely due to lack of DTD support.  Ignoring char.\n")
			continue
		
		if not dictEscapedChars.has_key(strUnescapedChar):
			dictEscapedChars[strUnescapedChar] = strEscapedChar
			
			
	
			
	#fdUTF8XML = StringIO.StringIO(strUTF8XML)
		
	global parser
	parser = expat.ParserCreate("utf-8") #uses utf-8 for parser input
	parser.returns_unicode  = 1
	parser.StartElementHandler = startElement
	parser.EndElementHandler = endElement
	parser.CharacterDataHandler = charData
	parser.StartCdataSectionHandler = startCData
	parser.EndCdataSectionHandler = endCData
	parser.CommentHandler = comment
	parser.ProcessingInstructionHandler = procInstruction
	parser.XmlDeclHandler = xmlDecl #requires Python 2.1
	parser.StartDoctypeDeclHandler = startDTD
	parser.EndDoctypeDeclHandler = endDTD
	#~ parser.ElementDeclHandler = elementDecl
	#~ parser.AttlistDeclHandler = attrDecl
	parser.EntityDeclHandler = entityDecl
	#~ parser.NotationDeclHandler = notationDecl
	#~ parser.DefaultHandler = defaultHandler
	
	#~ while 1:
		#~ chunk = fdUTF8XML.read(16384)
		#~ if not chunk:
			#~ parser.Parse("", 1)
			#~ break
		#~ else:
			#~ try:
				#~ parser.Parse(chunk)
			#~ except ExpatError:
				#~ print "Expat error"

	#parser.ParseFile(fdUTF8XML)
	parser.Parse(strUTF8XML, 1)
	
	

	sSLIP = strText
	return sSLIP #as Unicode
	parser = None
	strUnicodeXML = u""
	lsLines = []


if __name__ == "__main__":
	print """
Notice:  The command-line interface for converting SLiP files has been moved to the slip2xml.py and xml2slip.py scripts.

		"""